#########################################################################
# File Name: crawl.py
# Author: caochenglong
# mail: caochenglong@163.com
# Created Time: 2017-07-18 11:32:41
# Last modified:2017-07-18 11:32:44
#########################################################################
# !/usr/bin/python3
# _*_coding: utf-8_*_

from scrapy.commands import ScrapyCommand
from scrapy.crawler import CrawlerRunner
from scrapy.utils.conf import arglist_to_dict
from first.spiders.gamespider import GamespiderSpider
from first.spiders.tengxun import TengxunSpider
from twisted.internet import reactor
from scrapy.utils.log import configure_logging


class Command(ScrapyCommand):
    requires_project = True

    def syntax(self):
        return '[options]'

    def short_desc(self):
        return 'Runs all of the spiders'

    def add_options(self, parser):
        ScrapyCommand.add_options(self, parser)
        parser.add_option(
            "-a", dest="spargs",
            action="append", default=[], metavar="NAME=VALUE",
            help="set spider argument (may be repeated)")
        parser.add_option("-o", "--output", metavar="FILE",
                          help="dump scraped items \
                          into FILE (use - for stdout)")
        parser.add_option("-t", "--output-format", metavar="FORMAT",
                          help="format to use for dumping items with -o")

    def process_options(self, args, opts):
        ScrapyCommand.process_options(self, args, opts)
        try:
            opts.spargs = arglist_to_dict(opts.spargs)
        except ValueError:
            raise UsageError(
                "Invalid -a value, use -a NAME=VALUE", print_help=False)

    def run(self, args, opts):
        #  spider_loader = self.crawler_process.spider_loader
        #  for spidername in args or spider_loader.list():
        #      self.crawler_process.crawl(spidername, **opts.spargs)
        #  self.crawler_process.start()
        configure_logging()
        #  runner = CrawlerRunner()
        #  runner.crawl(GamespiderSpider, **opts.spargs)
        #  runner.crawl(TengxunSpider, **opts.spargs)
        self.crawler_process.crawl(GamespiderSpider, **opts.spargs)
        self.crawler_process.crawl(TengxunSpider, **opts.spargs)
        self.crawler_process.start()

        #  d = runner.join()
        #  d.addBoth(lambda _: reactor.stop())
        #  reactor.run()
